<!DOCTYPE html><html lang="zh-CN"><head><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="theme-color" content="#0078E7"><meta name="author" content="爱肖彤真是太好了"><meta name="copyright" content="爱肖彤真是太好了"><meta name="generator" content="Hexo 5.1.1"><meta name="theme" content="hexo-theme-yun"><title>Scrapy框架 | 工藤旧二の博客</title><link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Noto+Serif+SC:wght@900&amp;display=swap" media="none" onload="this.media='all'"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/star-markdown-css@0.1.19/dist/yun/yun-markdown.min.css"><script src="//at.alicdn.com/t/font_1140697_stqaphw3j4.js" async></script><script src="https://cdn.jsdelivr.net/npm/scrollreveal/dist/scrollreveal.min.js" defer></script><script>document.addEventListener("DOMContentLoaded", () => {
  [".post-card",".post-content img"].forEach((target)=> {
    ScrollReveal().reveal(target);
  })
});
</script><link rel="shortcut icon" type="image/svg+xml" href="/task/yun.ico"><link rel="mask-icon" href="/task/yun.ico" color="#0078E7"><link rel="alternate icon" href="/yun.ico"><link rel="preload" href="/task/css/hexo-theme-yun.css" as="style"><link rel="preload" href="/task/js/utils.js" as="script"><link rel="preload" href="/task/js/hexo-theme-yun.js" as="script"><link rel="prefetch" href="/task/js/sidebar.js" as="script"><link rel="preconnect" href="https://cdn.jsdelivr.net" crossorigin><link rel="stylesheet" href="/task/css/hexo-theme-yun.css"><link rel="alternate" href="/task/atom.xml" title="工藤旧二の博客" type="application/atom+xml"><script id="yun-config">
    const Yun = window.Yun || {};
    window.CONFIG = {"root":"/task/","title":"工藤旧二の博客","version":"0.9.7","anonymous_image":"https://cdn.jsdelivr.net/gh/YunYouJun/cdn/img/avatar/none.jpg","say":{"api":"https://v1.hitokoto.cn","hitokoto":true},"local_search":{"path":"/task/search.xml"},"fireworks":{"colors":["102, 167, 221","62, 131, 225","33, 78, 194"]}};
  </script><script>(function(){
  var bp = document.createElement('script');
  var curProtocol = window.location.protocol.split(':')[0];
  if (curProtocol === 'https') {
    bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
  }
  else {
    bp.src = 'http://push.zhanzhang.baidu.com/push.js';
  }
  var s = document.getElementsByTagName("script")[0];
  s.parentNode.insertBefore(bp, s);
})();</script><meta name="description" content="一、安装#1 pip3 install scrapy（mac，linux） #2 windows上（80%能成功，少部分人成功不了）     1、pip3 install wheel #安装后，便支持通过wheel文件安装软件，wheel文件官网：https:&#x2F;&#x2F;www.lfd.uci.edu&#x2F;~gohlke&#x2F;pythonlibs     3、pip3 install lxml     4、pip">
<meta property="og:type" content="article">
<meta property="og:title" content="Scrapy框架">
<meta property="og:url" content="http://huang_zhao.gitee.io/task/2020/04/22/python/%E7%88%AC%E8%99%AB/scrapy%E7%88%AC%E8%99%AB%E6%A1%86%E6%9E%B6/index.html">
<meta property="og:site_name" content="工藤旧二の博客">
<meta property="og:description" content="一、安装#1 pip3 install scrapy（mac，linux） #2 windows上（80%能成功，少部分人成功不了）     1、pip3 install wheel #安装后，便支持通过wheel文件安装软件，wheel文件官网：https:&#x2F;&#x2F;www.lfd.uci.edu&#x2F;~gohlke&#x2F;pythonlibs     3、pip3 install lxml     4、pip">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://gitee.com/huang_zhao/hz/raw/master/hz/1036857-20171109221422778-1731419400.png">
<meta property="article:published_time" content="2020-04-22T04:30:31.000Z">
<meta property="article:modified_time" content="2020-09-04T11:23:22.000Z">
<meta property="article:author" content="爱肖彤真是太好了">
<meta property="article:tag" content="爬虫">
<meta property="article:tag" content="python框架">
<meta property="article:tag" content="爬虫高级">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://gitee.com/huang_zhao/hz/raw/master/hz/1036857-20171109221422778-1731419400.png"><script src="/task/js/ui/mode.js"></script><link rel="stylesheet" href="/task/css/prism.css" type="text/css"></head><body><script defer src="https://cdn.jsdelivr.net/npm/animejs@latest/anime.min.js"></script><script defer src="/task/js/ui/fireworks.js"></script><canvas class="fireworks"></canvas><div class="container"><a class="sidebar-toggle hty-icon-button" id="menu-btn"><div class="hamburger hamburger--spin" type="button"><span class="hamburger-box"><span class="hamburger-inner"></span></span></div></a><div class="sidebar-toggle sidebar-overlay"></div><aside class="sidebar"><script src="/task/js/sidebar.js"></script><ul class="sidebar-nav"><li class="sidebar-nav-item sidebar-nav-toc hty-icon-button sidebar-nav-active" data-target="post-toc-wrap" title="文章目录"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-list-ordered"></use></svg></li><li class="sidebar-nav-item sidebar-nav-overview hty-icon-button" data-target="site-overview-wrap" title="站点概览"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-passport-line"></use></svg></li></ul><div class="sidebar-panel" id="site-overview-wrap"><div class="site-info fix-top"><a class="site-author-avatar" href="/task/about/" title="爱肖彤真是太好了"><img width="96" loading="lazy" src="/task/Yun.png" alt="爱肖彤真是太好了"></a><div class="site-author-name"><a href="/task/about/">爱肖彤真是太好了</a></div><a class="site-name" href="/task/about/site.html">工藤旧二の博客</a><sub class="site-subtitle"></sub><div class="site-desciption"></div></div><nav class="site-state"><a class="site-state-item hty-icon-button icon-home" href="/task/" title="首页"><span class="site-state-item-icon"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-home-4-line"></use></svg></span></a><div class="site-state-item"><a href="/task/archives/" title="归档"><span class="site-state-item-icon"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-archive-line"></use></svg></span><span class="site-state-item-count">57</span></a></div><div class="site-state-item"><a href="/task/categories/" title="分类"><span class="site-state-item-icon"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-folder-2-line"></use></svg></span><span class="site-state-item-count">2</span></a></div><div class="site-state-item"><a href="/task/tags/" title="标签"><span class="site-state-item-icon"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-price-tag-3-line"></use></svg></span><span class="site-state-item-count">50</span></a></div><a class="site-state-item hty-icon-button" target="_blank" rel="noopener" href="https://yun.yunyoujun.cn" title="文档"><span class="site-state-item-icon"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-settings-line"></use></svg></span></a></nav><hr style="margin-bottom:0.5rem"><div class="links-of-author"><a class="links-of-author-item hty-icon-button" rel="noopener" title="RSS" target="_blank" style="color:orange"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-rss-line"></use></svg></a><a class="links-of-author-item hty-icon-button" rel="noopener" title="QQ" target="_blank" style="color:#12B7F5"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-qq-line"></use></svg></a><a class="links-of-author-item hty-icon-button" rel="noopener" title="GitHub" target="_blank" style="color:#6e5494"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-github-line"></use></svg></a><a class="links-of-author-item hty-icon-button" rel="noopener" title="微博" target="_blank" style="color:#E6162D"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-weibo-line"></use></svg></a><a class="links-of-author-item hty-icon-button" rel="noopener" title="豆瓣" target="_blank" style="color:#007722"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-douban-line"></use></svg></a><a class="links-of-author-item hty-icon-button" rel="noopener" href="https://music.163.com/#/user/home?id=552858287" title="网易云音乐" target="_blank" style="color:#C20C0C"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-netease-cloud-music-line"></use></svg></a><a class="links-of-author-item hty-icon-button" rel="noopener" title="知乎" target="_blank" style="color:#0084FF"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-zhihu-line"></use></svg></a><a class="links-of-author-item hty-icon-button" rel="noopener" href="https://space.bilibili.com/15081363" title="哔哩哔哩" target="_blank" style="color:#FF8EB3"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-bilibili-line"></use></svg></a><a class="links-of-author-item hty-icon-button" rel="noopener" title="Twitter" target="_blank" style="color:#1da1f2"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-twitter-line"></use></svg></a><a class="links-of-author-item hty-icon-button" rel="noopener" title="Telegram Channel" target="_blank" style="color:#0088CC"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-telegram-line"></use></svg></a><a class="links-of-author-item hty-icon-button" rel="noopener" title="E-Mail" target="_blank" style="color:#8E71C1"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-mail-line"></use></svg></a><a class="links-of-author-item hty-icon-button" rel="noopener" title="Travelling" target="_blank" style="color:var(--hty-text-color)"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-send-plane-2-line"></use></svg></a></div><hr style="margin:0.5rem 1rem"><div class="links"><a class="links-item hty-icon-button" href="/task/links/" title="我的小伙伴们" style="color:dodgerblue"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-genderless-line"></use></svg></a></div><br><a class="links-item hty-icon-button" id="toggle-mode-btn" href="javascript:;" title="Mode" style="color: #f1cb64"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-contrast-2-line"></use></svg></a></div><div class="sidebar-panel sidebar-panel-active" id="post-toc-wrap"><div class="post-toc"><div class="post-toc-content"><ol class="toc"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%B8%80%E3%80%81%E5%AE%89%E8%A3%85"><span class="toc-text">一、安装</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%BA%8C%E3%80%81%E6%A1%86%E6%9E%B6%E7%AE%80%E4%BB%8B"><span class="toc-text">二、框架简介</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%B8%89%E3%80%81%E5%9F%BA%E6%9C%AC%E4%BD%BF%E7%94%A8"><span class="toc-text">三、基本使用</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#1-%E5%9F%BA%E6%9C%AC%E5%91%BD%E4%BB%A4"><span class="toc-text">1 基本命令</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#2-%E6%95%B0%E6%8D%AE%E8%A7%A3%E6%9E%90"><span class="toc-text">2 数据解析</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#3-%E6%95%B0%E6%8D%AE%E6%8C%81%E4%B9%85%E5%8C%96"><span class="toc-text">3 数据持久化</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E5%9B%9B%E3%80%81scrapy%E9%AB%98%E7%BA%A7"><span class="toc-text">四、scrapy高级</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#1-%E6%8F%90%E5%8D%87scrapy%E7%88%AC%E5%8F%96%E6%95%B0%E6%8D%AE%E7%9A%84%E6%95%88%E7%8E%87"><span class="toc-text">1 提升scrapy爬取数据的效率</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#2-scrapy%E7%9A%84%E4%B8%AD%E9%97%B4%E4%BB%B6%EF%BC%88%E4%B8%8B%E8%BD%BD%E4%B8%AD%E9%97%B4%E4%BB%B6%EF%BC%89"><span class="toc-text">2 scrapy的中间件（下载中间件）</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#3-selenium%E5%9C%A8scrapy%E4%B8%AD%E7%9A%84%E4%BD%BF%E7%94%A8%E6%B5%81%E7%A8%8B"><span class="toc-text">3 selenium在scrapy中的使用流程</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#4-%E5%88%86%E5%B8%83%E5%BC%8F%E7%88%AC%E8%99%AB%EF%BC%88scrapy-redis%EF%BC%89"><span class="toc-text">4 分布式爬虫（scrapy-redis）</span></a></li></ol></li></ol></div></div></div></aside><main class="sidebar-translate" id="content"><div id="post"><article class="post-block" itemscope itemtype="https://schema.org/Article"><link itemprop="mainEntityOfPage" href="http://huang_zhao.gitee.io/task/task/2020/04/22/python/%E7%88%AC%E8%99%AB/scrapy%E7%88%AC%E8%99%AB%E6%A1%86%E6%9E%B6/"><span hidden itemprop="author" itemscope itemtype="https://schema.org/Person"><meta itemprop="name" content="爱肖彤真是太好了"><meta itemprop="description"></span><span hidden itemprop="publisher" itemscope itemtype="https://schema.org/Organization"><meta itemprop="name" content="工藤旧二の博客"></span><header class="post-header"><h1 class="post-title" itemprop="name headline">Scrapy框架</h1><div class="post-meta"><div class="post-time" style="display:block"><span class="post-meta-item-icon"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-calendar-line"></use></svg></span> <time title="创建时间：2020-04-22 12:30:31" itemprop="dateCreated datePublished" datetime="2020-04-22T12:30:31+08:00">2020-04-22</time><span class="post-meta-divider">-</span><span class="post-meta-item-icon"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-calendar-2-line"></use></svg></span> <time title="修改时间：2020-09-04 19:23:22" itemprop="dateModified" datetime="2020-09-04T19:23:22+08:00">2020-09-04</time></div><span class="post-count"><span class="post-symbolcount"><span class="post-meta-item-icon" title="本文字数"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-file-word-line"></use></svg></span> <span title="本文字数">2.1k</span><span class="post-meta-divider">-</span><span class="post-meta-item-icon" title="阅读时长"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-timer-line"></use></svg></span> <span title="阅读时长">8m</span></span></span><span class="post-busuanzi"><span class="post-meta-divider">-</span><span class="post-meta-item-icon" title="阅读次数"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-eye-line"></use></svg> <span id="busuanzi_value_page_pv"></span></span></span><div class="post-classify"><span class="post-category"><span class="post-meta-item-icon"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-folder-line"></use></svg></span> <span itemprop="about" itemscope itemtype="https://schema.org/Thing"><a class="category" href="/task/categories/%E6%8A%80%E6%9C%AF%E5%88%86%E4%BA%AB/" style="--text-color:var(--hty-text-color)" itemprop="url" rel="index"><span itemprop="text">技术分享</span></a></span></span><span class="post-tag"><span class="post-meta-divider">-</span><a class="tag" href="/task/tags/%E7%88%AC%E8%99%AB/" style="--text-color:var(--hty-text-color)"><span class="post-meta-item-icon"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-price-tag-3-line"></use></svg></span><span class="tag-name">爬虫</span></a><a class="tag" href="/task/tags/python%E6%A1%86%E6%9E%B6/" style="--text-color:var(--hty-text-color)"><span class="post-meta-item-icon"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-price-tag-3-line"></use></svg></span><span class="tag-name">python框架</span></a><a class="tag" href="/task/tags/%E7%88%AC%E8%99%AB%E9%AB%98%E7%BA%A7/" style="--text-color:var(--hty-text-color)"><span class="post-meta-item-icon"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-price-tag-3-line"></use></svg></span><span class="tag-name">爬虫高级</span></a></span></div></div></header><section class="post-body" itemprop="articleBody"><div class="post-content markdown-body" style="--smc-primary:#0078E7;"><h2 id="一、安装"><a href="#一、安装" class="headerlink" title="一、安装"></a>一、安装</h2><pre class=" language-python"><code class="language-python"><span class="token comment" spellcheck="true">#1 pip3 install scrapy（mac，linux）</span>
<span class="token comment" spellcheck="true">#2 windows上（80%能成功，少部分人成功不了）</span>
    <span class="token number">1</span>、pip3 install wheel <span class="token comment" spellcheck="true">#安装后，便支持通过wheel文件安装软件，wheel文件官网：https://www.lfd.uci.edu/~gohlke/pythonlibs</span>
    <span class="token number">3</span>、pip3 install lxml
    <span class="token number">4</span>、pip3 install pyopenssl
    <span class="token number">5</span>、下载并安装pywin32：https<span class="token punctuation">:</span><span class="token operator">//</span>sourceforge<span class="token punctuation">.</span>net<span class="token operator">/</span>projects<span class="token operator">/</span>pywin32<span class="token operator">/</span>files<span class="token operator">/</span>pywin32<span class="token operator">/</span>
    <span class="token number">6</span>、下载twisted的wheel文件：http<span class="token punctuation">:</span><span class="token operator">//</span>www<span class="token punctuation">.</span>lfd<span class="token punctuation">.</span>uci<span class="token punctuation">.</span>edu<span class="token operator">/</span><span class="token operator">~</span>gohlke<span class="token operator">/</span>pythonlibs<span class="token operator">/</span><span class="token comment" spellcheck="true">#twisted</span>
    <span class="token number">7</span>、执行pip3 install 下载目录\Twisted<span class="token number">-17.9</span><span class="token punctuation">.</span><span class="token number">0</span><span class="token operator">-</span>cp36<span class="token operator">-</span>cp36m<span class="token operator">-</span>win_amd64<span class="token punctuation">.</span>whl
    <span class="token number">8</span>、pip3 install scrapy
<span class="token comment" spellcheck="true"># 3 就有scrapy命令</span>
    <span class="token operator">-</span>D<span class="token punctuation">:</span>\Python36\Scripts\scrapy<span class="token punctuation">.</span>exe  用于创建项目</code></pre>
<h2 id="二、框架简介"><a href="#二、框架简介" class="headerlink" title="二、框架简介"></a>二、框架简介</h2><p><strong>执行流程图</strong></p>
<p><img src="https://gitee.com/huang_zhao/hz/raw/master/hz/1036857-20171109221422778-1731419400.png" alt="1036857-20171109221422778-1731419400" loading="lazy"></p>
<ol>
<li><p>引擎(EGINE)</p>
<p>引擎负责控制系统所有组件之间的数据流，并在某些动作发生时触发事件。有关详细信息，请参见上面的数据流部分。</p>
<p>（scrapy的核心，用于转发数据或请求，如果是数据，转到管道去保存，如果是请求，转到调度器，如果来的是响应对象，转给爬虫）</p>
</li>
<li><p><strong>调度器(SCHEDULER)</strong><br>用来接受引擎发过来的请求, 压入队列中, 并在引擎再次请求的时候返回. 可以想像成一个URL的优先级队列, 由它来决定下一个要抓取的网址是什么, 同时去除重复的网址</p>
<p>（用于处理要爬取的url的顺序，深度优先/表示一条路走到黑，还是广度优先/表示同一层先爬下来，，去重）</p>
</li>
<li><p><strong>下载器(DOWLOADER)</strong><br>用于下载网页内容, 并将网页内容返回给EGINE，下载器是建立在twisted这个高效的异步模型上的</p>
<p>（向外要爬取的地址发送请求）</p>
</li>
<li><p><strong>爬虫(SPIDERS)</strong><br>SPIDERS是开发人员自定义的类，用来解析responses，并且提取items，或者发送新的请求</p>
</li>
<li><p><strong>项目管道(ITEM PIPLINES)</strong><br>在items被提取后负责处理它们，主要包括清理、验证、持久化（比如存到数据库）等操作</p>
<p>（需要在settings中配置）</p>
</li>
<li><p>下载器中间件(Downloader Middlewares)</p>
<p>位于Scrapy引擎和下载器之间，主要用来处理从EGINE传到DOWLOADER的请求request，已经从DOWNLOADER传到EGINE的响应response，你可用该中间件做以下几件事</p>
<ol>
<li>process a request just before it is sent to the Downloader (i.e. right before Scrapy sends the request to the website);</li>
<li>change received response before passing it to a spider;</li>
<li>send a new Request instead of passing received response to a spider;</li>
<li>pass response to a spider without fetching a web page;</li>
<li>silently drop some requests.</li>
</ol>
</li>
<li><p><strong>爬虫中间件(Spider Middlewares)</strong><br>位于EGINE和SPIDERS之间，主要工作是处理SPIDERS的输入（即responses）和输出（即requests）</p>
</li>
</ol>
<p>执行顺序：爬虫把要爬的url转发给引擎，引擎再转发给调度器，由调度器决定顺序，返回给引擎，引擎再把要爬取的url通过下载中间件（用于加个头，cookie啥的）向需要爬取的服务端发送请求，响应回来之后也通过下载中间件给引擎，引擎判断是响应数据之后，转发给爬虫，爬虫对数据进行处理，返回一个新的需要爬的地址（就继续上面的流程）或者需要保存的数据，数据部分由引擎转发给管道，保存数据</p>
<p><strong>目录介绍</strong></p>
<pre class=" language-python"><code class="language-python">    firstscrapy  <span class="token comment" spellcheck="true"># 项目名字</span>
        firstscrapy <span class="token comment" spellcheck="true"># 包</span>
            <span class="token operator">-</span>spiders <span class="token comment" spellcheck="true"># 所有的爬虫文件放在里面</span>
                <span class="token operator">-</span>baidu<span class="token punctuation">.</span>py <span class="token comment" spellcheck="true"># 一个个的爬虫（以后基本上都在这写东西）</span>
                <span class="token operator">-</span>chouti<span class="token punctuation">.</span>py
            <span class="token operator">-</span>middlewares<span class="token punctuation">.</span>py <span class="token comment" spellcheck="true"># 中间件（爬虫，下载中间件都写在这）</span>
            <span class="token operator">-</span>pipelines<span class="token punctuation">.</span>py   <span class="token comment" spellcheck="true"># 持久化相关写在这（items.py中类的对象）</span>
            <span class="token operator">-</span>main<span class="token punctuation">.</span>py        <span class="token comment" spellcheck="true"># 自己加的，执行爬虫</span>
            <span class="token operator">-</span>items<span class="token punctuation">.</span>py       <span class="token comment" spellcheck="true"># 一个一个的类，</span>
            <span class="token operator">-</span>settings<span class="token punctuation">.</span>py    <span class="token comment" spellcheck="true"># 配置文件</span>
        scrapy<span class="token punctuation">.</span>cfg          <span class="token comment" spellcheck="true"># 上线相关</span></code></pre>
<p><strong>settings参数介绍</strong></p>
<pre class=" language-python"><code class="language-python"><span class="token number">1</span> 默认情况，scrapy会去遵循爬虫协议
<span class="token number">2</span> 修改配置文件参数，强行爬取，不遵循协议
    <span class="token operator">-</span>ROBOTSTXT_OBEY <span class="token operator">=</span> <span class="token boolean">False</span>
<span class="token number">3</span> USER_AGENT <span class="token operator">=</span> <span class="token string">'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36'</span>
<span class="token number">4</span> LOG_LEVEL<span class="token operator">=</span><span class="token string">'ERROR'</span> <span class="token comment" spellcheck="true"># 运行的时候就不会把其他日志展示出来</span></code></pre>
<h2 id="三、基本使用"><a href="#三、基本使用" class="headerlink" title="三、基本使用"></a>三、基本使用</h2><h3 id="1-基本命令"><a href="#1-基本命令" class="headerlink" title="1 基本命令"></a>1 基本命令</h3><pre class=" language-python"><code class="language-python"><span class="token comment" spellcheck="true"># 创建项目</span>
scrapy startproject firstscrapy

<span class="token comment" spellcheck="true"># 创建爬虫</span>
scrapy genspider 爬虫名 爬虫地址
scrapy genspider chouti dig<span class="token punctuation">.</span>chouti<span class="token punctuation">.</span>com
<span class="token comment" spellcheck="true"># 一执行就会在spider文件夹下创建出一个py文件</span>

<span class="token comment" spellcheck="true"># 运行爬虫</span>
scrapy crawl chouti   <span class="token comment" spellcheck="true"># 带运行日志</span>
scrapy crawl chouti <span class="token operator">-</span><span class="token operator">-</span>nolog  <span class="token comment" spellcheck="true"># 不带日志</span>

<span class="token comment" spellcheck="true"># 支持右键执行爬虫</span>
<span class="token comment" spellcheck="true"># 在项目路径下新建一个main.py</span>
<span class="token keyword">from</span> scrapy<span class="token punctuation">.</span>cmdline <span class="token keyword">import</span> execute
execute<span class="token punctuation">(</span><span class="token punctuation">[</span><span class="token string">'scrapy'</span><span class="token punctuation">,</span><span class="token string">'crawl'</span><span class="token punctuation">,</span><span class="token string">'chouti'</span><span class="token punctuation">]</span><span class="token punctuation">)</span></code></pre>
<h3 id="2-数据解析"><a href="#2-数据解析" class="headerlink" title="2 数据解析"></a>2 数据解析</h3><pre class=" language-python"><code class="language-python"><span class="token comment" spellcheck="true">#xpath：</span>
    <span class="token operator">-</span>response<span class="token punctuation">.</span>xpath<span class="token punctuation">(</span><span class="token string">'//a[contains(@class,"link-title")]/text()'</span><span class="token punctuation">)</span><span class="token punctuation">.</span>extract<span class="token punctuation">(</span><span class="token punctuation">)</span>  <span class="token comment" spellcheck="true"># 取文本</span>
    <span class="token operator">-</span>response<span class="token punctuation">.</span>xpath<span class="token punctuation">(</span><span class="token string">'//a[contains(@class,"link-title")]/@href'</span><span class="token punctuation">)</span><span class="token punctuation">.</span>extract<span class="token punctuation">(</span><span class="token punctuation">)</span>  <span class="token comment" spellcheck="true">#取属性</span>
<span class="token comment" spellcheck="true">#css</span>
    <span class="token operator">-</span>response<span class="token punctuation">.</span>css<span class="token punctuation">(</span><span class="token string">'.link-title::text'</span><span class="token punctuation">)</span><span class="token punctuation">.</span>extract<span class="token punctuation">(</span><span class="token punctuation">)</span>  <span class="token comment" spellcheck="true"># 取文本</span>
    <span class="token operator">-</span>response<span class="token punctuation">.</span>css<span class="token punctuation">(</span><span class="token string">'.link-title::attr(href)'</span><span class="token punctuation">)</span><span class="token punctuation">.</span>extract_first<span class="token punctuation">(</span><span class="token punctuation">)</span>  <span class="token comment" spellcheck="true"># 取属性</span>
</code></pre>
<h3 id="3-数据持久化"><a href="#3-数据持久化" class="headerlink" title="3 数据持久化"></a>3 数据持久化</h3><p><strong>items.py</strong></p>
<pre class=" language-python"><code class="language-python"><span class="token comment" spellcheck="true"># 持久化匹配的字段</span>
<span class="token keyword">class</span> <span class="token class-name">ChoutiItem</span><span class="token punctuation">(</span>scrapy<span class="token punctuation">.</span>Item<span class="token punctuation">)</span><span class="token punctuation">:</span>
    <span class="token comment" spellcheck="true"># define the fields for your item here like:</span>
    title <span class="token operator">=</span> scrapy<span class="token punctuation">.</span>Field<span class="token punctuation">(</span><span class="token punctuation">)</span>
    laiyuan <span class="token operator">=</span> scrapy<span class="token punctuation">.</span>Field<span class="token punctuation">(</span><span class="token punctuation">)</span></code></pre>
<p><strong>spiders/chouti.py</strong></p>
<pre class=" language-python"><code class="language-python"><span class="token keyword">import</span> scrapy


<span class="token keyword">class</span> <span class="token class-name">ChoutiSpider</span><span class="token punctuation">(</span>scrapy<span class="token punctuation">.</span>Spider<span class="token punctuation">)</span><span class="token punctuation">:</span>
    name <span class="token operator">=</span> <span class="token string">'chouti'</span>
    allowed_domains <span class="token operator">=</span> <span class="token punctuation">[</span><span class="token string">'dig.chouti.com'</span><span class="token punctuation">]</span>
    start_urls <span class="token operator">=</span> <span class="token punctuation">[</span><span class="token string">'http://dig.chouti.com/'</span><span class="token punctuation">]</span>

    <span class="token keyword">def</span> <span class="token function">parse</span><span class="token punctuation">(</span>self<span class="token punctuation">,</span> response<span class="token punctuation">)</span><span class="token punctuation">:</span>
        <span class="token comment" spellcheck="true"># 返回要爬取的页面，或者返回要保存的数据</span>
        <span class="token comment" spellcheck="true"># 下面两者效果相同</span>
        <span class="token keyword">from</span> pachong<span class="token punctuation">.</span>items <span class="token keyword">import</span> ChoutiItem
        item <span class="token operator">=</span> ChoutiItem<span class="token punctuation">(</span><span class="token punctuation">)</span>
        div_list <span class="token operator">=</span> response<span class="token punctuation">.</span>css<span class="token punctuation">(</span><span class="token string">'.link-con .link-item'</span><span class="token punctuation">)</span>
        <span class="token comment" spellcheck="true"># div_list = response.xpath('//div[contains(@class,"link-item")]')</span>
        <span class="token keyword">for</span> div <span class="token keyword">in</span> div_list<span class="token punctuation">:</span>
            title <span class="token operator">=</span> div<span class="token punctuation">.</span>css<span class="token punctuation">(</span><span class="token string">'.link-title::text'</span><span class="token punctuation">)</span><span class="token punctuation">.</span>extract_first<span class="token punctuation">(</span><span class="token punctuation">)</span>
            laiyuan <span class="token operator">=</span> div<span class="token punctuation">.</span>css<span class="token punctuation">(</span><span class="token string">'.link-from::text'</span><span class="token punctuation">)</span><span class="token punctuation">.</span>extract_first<span class="token punctuation">(</span><span class="token punctuation">)</span>
            <span class="token comment" spellcheck="true"># title = div.xpath('//a[contains(@class,"link-title")]/text()').extract()</span>
            <span class="token keyword">if</span> <span class="token operator">not</span> laiyuan<span class="token punctuation">:</span>
                laiyuan <span class="token operator">=</span> <span class="token string">''</span>
            <span class="token keyword">print</span><span class="token punctuation">(</span>title<span class="token punctuation">,</span>laiyuan<span class="token punctuation">)</span>
            item<span class="token punctuation">[</span><span class="token string">'title'</span><span class="token punctuation">]</span> <span class="token operator">=</span> title
            item<span class="token punctuation">[</span><span class="token string">'laiyuan'</span><span class="token punctuation">]</span> <span class="token operator">=</span> laiyuan
            <span class="token keyword">yield</span> item</code></pre>
<p><strong>pipelines.py</strong></p>
<pre class=" language-python"><code class="language-python"><span class="token keyword">import</span> pymysql
<span class="token keyword">class</span> <span class="token class-name">PachongPipeline</span><span class="token punctuation">:</span>
    <span class="token keyword">def</span> <span class="token function">open_spider</span><span class="token punctuation">(</span>self<span class="token punctuation">,</span>spider<span class="token punctuation">)</span><span class="token punctuation">:</span>
        self<span class="token punctuation">.</span>conn <span class="token operator">=</span> pymysql<span class="token punctuation">.</span>connect<span class="token punctuation">(</span>host<span class="token operator">=</span><span class="token string">'127.0.0.1'</span><span class="token punctuation">,</span> user<span class="token operator">=</span><span class="token string">'root'</span><span class="token punctuation">,</span> password<span class="token operator">=</span><span class="token string">"root"</span><span class="token punctuation">,</span>
                 database<span class="token operator">=</span><span class="token string">'pachong'</span><span class="token punctuation">,</span> port<span class="token operator">=</span><span class="token number">3306</span><span class="token punctuation">)</span>

    <span class="token keyword">def</span> <span class="token function">process_item</span><span class="token punctuation">(</span>self<span class="token punctuation">,</span> item<span class="token punctuation">,</span> spider<span class="token punctuation">)</span><span class="token punctuation">:</span>
        cursor <span class="token operator">=</span> self<span class="token punctuation">.</span>conn<span class="token punctuation">.</span>cursor<span class="token punctuation">(</span><span class="token punctuation">)</span>
        sql <span class="token operator">=</span> <span class="token string">'insert into chouti (title,laiyuan)values(%s,%s)'</span>
        cursor<span class="token punctuation">.</span>execute<span class="token punctuation">(</span>sql<span class="token punctuation">,</span><span class="token punctuation">[</span>item<span class="token punctuation">[</span><span class="token string">'title'</span><span class="token punctuation">]</span><span class="token punctuation">,</span>item<span class="token punctuation">[</span><span class="token string">'laiyuan'</span><span class="token punctuation">]</span><span class="token punctuation">]</span><span class="token punctuation">)</span>
        self<span class="token punctuation">.</span>conn<span class="token punctuation">.</span>commit<span class="token punctuation">(</span><span class="token punctuation">)</span>
        <span class="token keyword">return</span> item

    <span class="token keyword">def</span> <span class="token function">close_spider</span><span class="token punctuation">(</span>self<span class="token punctuation">,</span>spider<span class="token punctuation">)</span><span class="token punctuation">:</span>
        self<span class="token punctuation">.</span>conn<span class="token punctuation">.</span>close<span class="token punctuation">(</span><span class="token punctuation">)</span></code></pre>
<p><strong>settings.py</strong></p>
<pre class=" language-python"><code class="language-python">ITEM_PIPELINES <span class="token operator">=</span> <span class="token punctuation">{</span>
   <span class="token string">'pachong.pipelines.PachongPipeline'</span><span class="token punctuation">:</span> <span class="token number">300</span><span class="token punctuation">,</span>
<span class="token punctuation">}</span>
<span class="token comment" spellcheck="true"># 配置优先级</span></code></pre>
<h2 id="四、scrapy高级"><a href="#四、scrapy高级" class="headerlink" title="四、scrapy高级"></a>四、scrapy高级</h2><h3 id="1-提升scrapy爬取数据的效率"><a href="#1-提升scrapy爬取数据的效率" class="headerlink" title="1 提升scrapy爬取数据的效率"></a>1 提升scrapy爬取数据的效率</h3><pre class=" language-python"><code class="language-python"><span class="token operator">-</span> 在配置文件中进行相关的配置即可<span class="token punctuation">:</span><span class="token punctuation">(</span>默认还有一套setting<span class="token punctuation">)</span>
<span class="token comment" spellcheck="true">#1 增加并发：</span>
默认scrapy开启的并发线程为<span class="token number">32</span>个，可以适当进行增加。在settings配置文件中修改CONCURRENT_REQUESTS <span class="token operator">=</span> <span class="token number">100</span>值为<span class="token number">100</span><span class="token punctuation">,</span>并发设置成了为<span class="token number">100</span>。
<span class="token comment" spellcheck="true">#2 降低日志级别：</span>
在运行scrapy时，会有大量日志信息的输出，为了减少CPU的使用率。可以设置log输出信息为INFO或者ERROR即可。在配置文件中编写：LOG_LEVEL <span class="token operator">=</span> ‘INFO’
<span class="token comment" spellcheck="true"># 3 禁止cookie：</span>
如果不是真的需要cookie，则在scrapy爬取数据时可以禁止cookie从而减少CPU的使用率，提升爬取效率。在配置文件中编写：COOKIES_ENABLED <span class="token operator">=</span> <span class="token boolean">False</span>
<span class="token comment" spellcheck="true"># 4禁止重试：</span>
对失败的HTTP进行重新请求（重试）会减慢爬取速度，因此可以禁止重试。在配置文件中编写：RETRY_ENABLED <span class="token operator">=</span> <span class="token boolean">False</span>
<span class="token comment" spellcheck="true"># 5 减少下载超时：</span>
如果对一个非常慢的链接进行爬取，减少下载超时可以能让卡住的链接快速被放弃，从而提升效率。在配置文件中进行编写：DOWNLOAD_TIMEOUT <span class="token operator">=</span> <span class="token number">10</span> 超时时间为10s</code></pre>
<h3 id="2-scrapy的中间件（下载中间件）"><a href="#2-scrapy的中间件（下载中间件）" class="headerlink" title="2 scrapy的中间件（下载中间件）"></a>2 scrapy的中间件（下载中间件）</h3><pre class=" language-python"><code class="language-python"><span class="token comment" spellcheck="true"># 1 都写在middlewares.py</span>
<span class="token comment" spellcheck="true"># 2 爬虫中间件</span>
<span class="token comment" spellcheck="true"># 3 下载中间件</span>
<span class="token comment" spellcheck="true"># 4 要生效，一定要配置，配置文件</span>


<span class="token comment" spellcheck="true"># 下载中间件</span>
<span class="token operator">-</span>process_request：返回不同的对象，后续处理不同（加代理<span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span>）
          <span class="token comment" spellcheck="true"># 1 更换请求头</span>
        <span class="token comment" spellcheck="true"># print(type(request.headers))</span>
        <span class="token comment" spellcheck="true"># print(request.headers)</span>
        <span class="token comment" spellcheck="true">#</span>
        <span class="token comment" spellcheck="true"># from scrapy.http.headers import Headers</span>
        <span class="token comment" spellcheck="true"># request.headers['User-Agent']=''</span>

        <span class="token comment" spellcheck="true"># 2 加cookie ---cookie池</span>
        <span class="token comment" spellcheck="true"># 假设你你已经搭建好cookie 池了，</span>
        <span class="token comment" spellcheck="true"># print('00000--',request.cookies)</span>
        <span class="token comment" spellcheck="true"># request.cookies={'username':'asdfasdf'}</span>

        <span class="token comment" spellcheck="true"># 3 加代理</span>
        <span class="token comment" spellcheck="true"># print(request.meta)</span>
        <span class="token comment" spellcheck="true"># request.meta['download_timeout'] = 20</span>
        <span class="token comment" spellcheck="true"># request.meta["proxy"] = 'http://27.188.62.3:8060'</span>
<span class="token operator">-</span>process_response：返回不同的对象，后续处理不同
<span class="token operator">-</span> process_exception
<span class="token keyword">def</span> <span class="token function">process_exception</span><span class="token punctuation">(</span>self<span class="token punctuation">,</span> request<span class="token punctuation">,</span> exception<span class="token punctuation">,</span> spider<span class="token punctuation">)</span><span class="token punctuation">:</span>
        <span class="token keyword">print</span><span class="token punctuation">(</span><span class="token string">'xxxx'</span><span class="token punctuation">)</span>
        <span class="token comment" spellcheck="true"># 不允许直接改url</span>
        <span class="token comment" spellcheck="true"># request.url='https://www.baidu.com'</span>
        <span class="token keyword">from</span> scrapy <span class="token keyword">import</span> Request
        request<span class="token operator">=</span>Request<span class="token punctuation">(</span>url<span class="token operator">=</span><span class="token string">'https://www.baidu.com'</span><span class="token punctuation">,</span>callback<span class="token operator">=</span>spider<span class="token punctuation">.</span>parser<span class="token punctuation">)</span>
        <span class="token keyword">return</span> request

</code></pre>
<h3 id="3-selenium在scrapy中的使用流程"><a href="#3-selenium在scrapy中的使用流程" class="headerlink" title="3 selenium在scrapy中的使用流程"></a>3 selenium在scrapy中的使用流程</h3><pre class=" language-python"><code class="language-python"><span class="token comment" spellcheck="true"># 当前爬虫用的selenium是同一个</span>

<span class="token comment" spellcheck="true"># 1 在爬虫中初始化webdriver对象</span>
    <span class="token keyword">from</span> selenium <span class="token keyword">import</span> webdriver
    <span class="token keyword">class</span> <span class="token class-name">CnblogSpider</span><span class="token punctuation">(</span>scrapy<span class="token punctuation">.</span>Spider<span class="token punctuation">)</span><span class="token punctuation">:</span>
        name <span class="token operator">=</span> <span class="token string">'cnblog'</span>
        <span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span>
 bro<span class="token operator">=</span>webdriver<span class="token punctuation">.</span>Chrome<span class="token punctuation">(</span>executable_path<span class="token operator">=</span><span class="token string">'../chromedriver.exe'</span><span class="token punctuation">)</span>
<span class="token comment" spellcheck="true"># 2 在中间件中使用（process_request）</span>
spider<span class="token punctuation">.</span>bro<span class="token punctuation">.</span>get<span class="token punctuation">(</span><span class="token string">'https://dig.chouti.com/'</span><span class="token punctuation">)</span>   response<span class="token operator">=</span>HtmlResponse<span class="token punctuation">(</span>url<span class="token operator">=</span><span class="token string">'https://dig.chouti.com/'</span><span class="token punctuation">,</span>body<span class="token operator">=</span>spider<span class="token punctuation">.</span>bro<span class="token punctuation">.</span>page_source<span class="token punctuation">.</span>encode<span class="token punctuation">(</span><span class="token string">'utf-8'</span><span class="token punctuation">)</span><span class="token punctuation">,</span>request<span class="token operator">=</span>request<span class="token punctuation">)</span>
    <span class="token keyword">return</span> response

<span class="token comment" spellcheck="true"># 3 在爬虫中关闭</span>
    <span class="token keyword">def</span> <span class="token function">close</span><span class="token punctuation">(</span>self<span class="token punctuation">,</span> reason<span class="token punctuation">)</span><span class="token punctuation">:</span>
        <span class="token keyword">print</span><span class="token punctuation">(</span><span class="token string">"我结束了"</span><span class="token punctuation">)</span>
        self<span class="token punctuation">.</span>bro<span class="token punctuation">.</span>close<span class="token punctuation">(</span><span class="token punctuation">)</span></code></pre>
<h3 id="4-分布式爬虫（scrapy-redis）"><a href="#4-分布式爬虫（scrapy-redis）" class="headerlink" title="4 分布式爬虫（scrapy-redis）"></a>4 分布式爬虫（scrapy-redis）</h3><pre class=" language-python"><code class="language-python"><span class="token comment" spellcheck="true"># 1 pip3 install scrapy-redis</span>
<span class="token comment" spellcheck="true"># 2 原来继承Spider，现在继承RedisSpider</span>
<span class="token comment" spellcheck="true"># 3 不能写start_urls = ['https:/www.cnblogs.com/']</span>
<span class="token comment" spellcheck="true"># 4 需要写redis_key = 'myspider:start_urls'</span>
<span class="token comment" spellcheck="true"># 5 setting中配置：</span>

<span class="token comment" spellcheck="true"># redis的连接</span>
REDIS_HOST <span class="token operator">=</span> <span class="token string">'localhost'</span>                            <span class="token comment" spellcheck="true"># 主机名</span>
REDIS_PORT <span class="token operator">=</span> <span class="token number">6379</span>                                   <span class="token comment" spellcheck="true"># 端口</span>
    <span class="token comment" spellcheck="true"># 使用scrapy-redis的去重</span>
DUPEFILTER_CLASS <span class="token operator">=</span> <span class="token string">"scrapy_redis.dupefilter.RFPDupeFilter"</span>
<span class="token comment" spellcheck="true"># 使用scrapy-redis的Scheduler</span>
<span class="token comment" spellcheck="true"># 分布式爬虫的配置</span>
SCHEDULER <span class="token operator">=</span> <span class="token string">"scrapy_redis.scheduler.Scheduler"</span>
<span class="token comment" spellcheck="true"># 持久化的可以配置，也可以不配置</span>
ITEM_PIPELINES <span class="token operator">=</span> <span class="token punctuation">{</span>
   <span class="token string">'scrapy_redis.pipelines.RedisPipeline'</span><span class="token punctuation">:</span> <span class="token number">299</span>
<span class="token punctuation">}</span>


<span class="token comment" spellcheck="true"># 9现在要让爬虫运行起来，需要去redis中以myspider:start_urls为key，插入一个起始地址lpush myspider:start_urls https://www.cnblogs.com/</span></code></pre>
</div><div id="reward-container"><span class="hty-icon-button button-glow" id="reward-button" title="打赏" onclick="var qr = document.getElementById(&quot;qr&quot;); qr.style.display = (qr.style.display === &quot;none&quot;) ? &quot;block&quot; : &quot;none&quot;;"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-hand-coin-line"></use></svg></span><div id="reward-comment">点我就给你看点有意思的</div><div id="qr" style="display:none;"><div style="display:inline-block"><a target="_blank" rel="noopener" href="https://gitee.com/huang_zhao/hz/raw/master/hz/alipay.png"><img loading="lazy" src="https://gitee.com/huang_zhao/hz/raw/master/hz/alipay.png" alt="支付宝" title="支付宝"></a><div><span style="color:#00A3EE"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-alipay-line"></use></svg></span></div></div><div style="display:inline-block"><a target="_blank" rel="noopener" href="https://gitee.com/huang_zhao/hz/raw/master/hz/qqpay.jpg"><img loading="lazy" src="https://gitee.com/huang_zhao/hz/raw/master/hz/qqpay.jpg" alt="QQ 支付" title="QQ 支付"></a><div><span style="color:#12B7F5"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-qq-line"></use></svg></span></div></div><div style="display:inline-block"><a target="_blank" rel="noopener" href="https://gitee.com/huang_zhao/hz/raw/master/hz/wxpay.jpg"><img loading="lazy" src="https://gitee.com/huang_zhao/hz/raw/master/hz/wxpay.jpg" alt="微信支付" title="微信支付"></a><div><span style="color:#2DC100"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-wechat-pay-line"></use></svg></span></div></div></div></div><ul class="post-copyright"><li class="post-copyright-author"><strong>本文作者：</strong>爱肖彤真是太好了</li><li class="post-copyright-link"><strong>本文链接：</strong><a href="http://huang_zhao.gitee.io/task/2020/04/22/python/%E7%88%AC%E8%99%AB/scrapy%E7%88%AC%E8%99%AB%E6%A1%86%E6%9E%B6/" title="Scrapy框架">http://huang_zhao.gitee.io/task/2020/04/22/python/%E7%88%AC%E8%99%AB/scrapy%E7%88%AC%E8%99%AB%E6%A1%86%E6%9E%B6/</a></li><li class="post-copyright-license"><strong>版权声明：</strong>本博客所有文章除特别声明外，均默认采用 <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/deed.zh" target="_blank" rel="noopener" title="CC BY-NC-SA 4.0 "><svg class="icon"><use xlink:href="#icon-creative-commons-line"></use></svg><svg class="icon"><use xlink:href="#icon-creative-commons-by-line"></use></svg><svg class="icon"><use xlink:href="#icon-creative-commons-nc-line"></use></svg><svg class="icon"><use xlink:href="#icon-creative-commons-sa-line"></use></svg></a> 许可协议。</li></ul></section></article><div class="post-nav"><div class="post-nav-item"><a class="post-nav-prev" href="/task/2020/04/23/%E6%95%B0%E6%8D%AE%E5%BA%93/redis/Redis%E7%AE%A1%E9%81%93%EF%BC%8C%E4%BD%8D%E5%9B%BE%EF%BC%8Cgeo%EF%BC%8CHyperLogLog%EF%BC%8C%E5%8F%91%E5%B8%83%E8%AE%A2%E9%98%85/" rel="prev" title="Redis管道，位图，geo，HyperLogLog，发布订阅"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-arrow-left-s-line"></use></svg><span class="post-nav-text">Redis管道，位图，geo，HyperLogLog，发布订阅</span></a></div><div class="post-nav-item"><a class="post-nav-next" href="/task/2020/04/21/python/%E7%88%AC%E8%99%AB/BeautifulSoup%E6%A8%A1%E5%9D%97%E7%9A%84%E4%BD%BF%E7%94%A8/" rel="next" title="BeautifulSoup模块的使用"><span class="post-nav-text">BeautifulSoup模块的使用</span><svg class="icon" aria-hidden="true"><use xlink:href="#icon-arrow-right-s-line"></use></svg></a></div></div></div></main><footer class="sidebar-translate" id="footer"><div class="copyright"><span>&copy; 2019 – 2022 </span><span class="with-love" id="animate"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-cloud-line"></use></svg></span><span class="author"> 爱肖彤真是太好了</span></div><div class="powered"><span>由 <a href="https://hexo.io" target="_blank" rel="noopener">Hexo</a> 驱动 v5.1.1</span><span class="footer-separator">|</span><span>主题 - <a rel="noopener" href="https://github.com/YunYouJun/hexo-theme-yun" target="_blank"><span>Yun</span></a> v0.9.7</span></div><div class="live_time"><span>本博客已萌萌哒地运行</span><span id="display_live_time"></span><span class="moe-text">(●'◡'●)</span><script>function blog_live_time() {
  window.setTimeout(blog_live_time, 1000);
  const start = new Date('2019-04-12T00:00:00');
  const now = new Date();
  const timeDiff = (now.getTime() - start.getTime());
  const msPerMinute = 60 * 1000;
  const msPerHour = 60 * msPerMinute;
  const msPerDay = 24 * msPerHour;
  const passDay = Math.floor(timeDiff / msPerDay);
  const passHour = Math.floor((timeDiff % msPerDay) / 60 / 60 / 1000);
  const passMinute = Math.floor((timeDiff % msPerHour) / 60 / 1000);
  const passSecond = Math.floor((timeDiff % msPerMinute) / 1000);
  display_live_time.innerHTML = " " + passDay + " 天 " + passHour + " 小时 " + passMinute + " 分 " + passSecond + " 秒";
}
blog_live_time();
</script></div><div id="busuanzi"><script async src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script><span id="busuanzi_container_site_uv" title="总访客量"><span><svg class="icon" aria-hidden="true"><use xlink:href="#icon-user-line"></use></svg></span><span id="busuanzi_value_site_uv"></span></span><span class="footer-separator">|</span><span id="busuanzi_container_site_pv" title="总访问量"><span><svg class="icon" aria-hidden="true"><use xlink:href="#icon-eye-line"></use></svg></span><span id="busuanzi_value_site_pv"></span></span></div></footer><a class="hty-icon-button" id="goUp" aria-label="back-to-top" href="#"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-arrow-up-s-line"></use></svg><svg class="progress-circle-container" viewBox="0 0 100 100"><circle class="progress-circle" id="progressCircle" cx="50" cy="50" r="48" fill="none" stroke="#0078E7" stroke-width="2" stroke-linecap="round"></circle></svg></a><a class="popup-trigger hty-icon-button icon-search" id="search" href="javascript:;" title="搜索"><span class="site-state-item-icon"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-search-line"></use></svg></span></a><script>window.addEventListener("DOMContentLoaded", () => {
  // Handle and trigger popup window
  document.querySelector(".popup-trigger").addEventListener("click", () => {
    document.querySelector(".popup").classList.add("show");
    setTimeout(() => {
      document.querySelector(".search-input").focus();
    }, 100);
  });

  // Monitor main search box
  const onPopupClose = () => {
    document.querySelector(".popup").classList.remove("show");
  };

  document.querySelector(".popup-btn-close").addEventListener("click", () => {
    onPopupClose();
  });

  window.addEventListener("keyup", event => {
    if (event.key === "Escape") {
      onPopupClose();
    }
  });
});
</script><script src="/task/js/search/local-search.js" defer></script><div class="popup search-popup"><div class="search-header"><span class="popup-btn-close close-icon hty-icon-button"><svg class="icon" aria-hidden="true"><use xlink:href="#icon-close-line"></use></svg></span></div><div class="search-input-container"><input class="search-input" id="local-search-input" type="text" placeholder="搜索..." value=""></div><div id="local-search-result"></div></div></div><script defer src="/task/js/utils.js"></script><script defer src="/task/js/hexo-theme-yun.js"></script></body></html>